package com.jobsika.solr.crawler;

import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;
import java.util.Vector;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.core.CoreContainer;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

import com.jobsika.solr.domain.Post;
/**
 * Google Finance Parser
 * @author Bernie
 * java -cp .:../lib/libthrift-r917130.jar:../lib/slf4j-log4j12-1.5.8.jar:../lib/slf4j-api-1.5.8.jar:../lib/log4j-1.2.14.jar:../lib/apache-cassandra-0.6.1.jar com.jobsika.data.Craigslist
 */
public class Craigslist extends DefaultHandler {
	private static SimpleDateFormat fmt;
	private static SimpleDateFormat wfmt;
	private TimeZone tz;
	private static Vector<Post> posts;
	private Post lastPost;
	private boolean getTitle;
	private boolean getDesc;
	private boolean getDate;
	
	public Craigslist() {
		fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
		wfmt = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z");
		tz = TimeZone.getTimeZone("America/Los_Angeles");
		fmt.setTimeZone(tz);
		wfmt.setTimeZone(tz);
		posts = new Vector<Post>();
	}
	
	/**
	 * XML <VAL>xml</xml>
	 */
	public void startElement(String uri, String localName, String qName, Attributes attr) throws SAXException
    {
		if (qName.equalsIgnoreCase("item")) {
			String link = attr.getValue(0);
			try {
				URL url = new URL(link);
				lastPost = new Post();
				lastPost.setId(url.toString());
			} catch (MalformedURLException e) {
			}
		}
		if (qName.equalsIgnoreCase("title")) {
			getTitle = true;
		}
		if (qName.equalsIgnoreCase("description")) {
			getDesc = true;
		}
		if (qName.equalsIgnoreCase("dc:date")) {
			getDate = true;
		}
		
    }

	/**
	 * XML <xml>VALUE</xml>
	 */
	public void characters(char ch[], int start, int length) throws SAXException {
		String value = new String(ch, start, length);
		if (getTitle) {
			String location = value;
			String title = value;
			int str = value.lastIndexOf('(');
			int end = value.lastIndexOf(')');
			if (str > 0 && end > 0) {
				location = value.substring(str+1, end);
				title = value.substring(0, str);
			}
			if (lastPost != null) {
				lastPost.setTitle(ServiceConstants.escapeHTML(title));
				if (str > 0 && end > 0) {
					location = DataUtil.extractCalifornia(location);
					lastPost.setLocation(ServiceConstants.escapeHTML(location));
				}
			}
			getTitle = false;
		}
		if (getDesc) {
			if (lastPost != null) {
				lastPost.setBody(ServiceConstants.escapeHTML(value));
			}
			getDesc = false;
		}
		if (getDate) {
			if (lastPost != null) {
				try {
					value = value.substring(0, value.lastIndexOf(':')) + value.substring(value.lastIndexOf(':')+1);
					Date date = fmt.parse(value);
					lastPost.setDate(date);
					if (lastPost.getTitle() != null && 
							lastPost.getDate() != null && 
							lastPost.getBody() != null && 
							lastPost.getId() != null && 
							lastPost.getLocation() != null) {
						posts.add(lastPost);
					}
				} catch (ParseException e) {
					e.printStackTrace();
				}
				
			}
			getDate = false;
			lastPost = null;
		}
	}

	/**
	 * XML <xml>xml</VAL>
	 */
	public void endElement(String uri, String lName, String qName)  throws SAXException
	{
	}
	
	//"http://sfbay.craigslist.org/sby/sof/index.rss"
	public static void parseURL(EmbeddedSolrServer server, SAXParser parser, DefaultHandler sx, String myURL) throws Exception {
		try {
			URL url = new URL(myURL);
			URLConnection conn = url.openConnection();
			parser.parse(conn.getInputStream(), sx);
			for (Post p : posts) {
				server.addBean(p);
			}
		    server.commit();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void main(String[] argv) throws Exception {
		CoreContainer.Initializer initializer = new CoreContainer.Initializer();
		CoreContainer coreContainer;
		coreContainer = initializer.initialize();
		EmbeddedSolrServer server = new EmbeddedSolrServer(coreContainer, "");
		SAXParserFactory factory = SAXParserFactory.newInstance();
		SAXParser parser = factory.newSAXParser();
		DefaultHandler sx = new Craigslist();
		String[] categories = {"acc", "ofc", "egr", "med", "sci", "bus", "csr", "edu", "fbh", "lab", "gov", "hum", "eng", "lgl", "mnu", "mar", "hea", "npo", "rej", "ret", "sls", "spa", "sec", "trd", "sad", "tch", "trp", "tfr", "web", "wri"};
		for (String category : categories) {
			parseURL(server, parser, sx, "http://sfbay.craigslist.org/sby/"+category+"/index.rss");
		}
	}
}